<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
<html xmlns="http://www.w3.org/1999/xhtml">
<head>
<meta http-equiv="Content-Type" content="text/xhtml;charset=UTF-8"/>
<meta http-equiv="X-UA-Compatible" content="IE=9"/>
<title>Jacobi: Host.c File Reference</title>

<link href="tabs.css" rel="stylesheet" type="text/css"/>
<link href="doxygen.css" rel="stylesheet" type="text/css" />

<link href="search/search.css" rel="stylesheet" type="text/css"/>
<script type="text/javascript" src="jquery.js"></script>
<script type="text/javascript" src="search/search.js"></script>
<script type="text/javascript">
  $(document).ready(function() { searchBox.OnSelectItem(0); });
</script>

</head>
<body>
<div id="top"><!-- do not remove this div! -->


<div id="titlearea">
<table cellspacing="0" cellpadding="0">
 <tbody>
 <tr style="height: 56px;">
  
  
  <td style="padding-left: 0.5em;">
   <div id="projectname">Jacobi
   
   </div>
   
  </td>
  
  
  
 </tr>
 </tbody>
</table>
</div>

<!-- Generated by Doxygen 1.7.6.1 -->
<script type="text/javascript">
var searchBox = new SearchBox("searchBox", "search",false,'Search');
</script>
  <div id="navrow1" class="tabs">
    <ul class="tablist">
      <li><a href="index.html"><span>Main&#160;Page</span></a></li>
      <li class="current"><a href="files.html"><span>Files</span></a></li>
      <li>
        <div id="MSearchBox" class="MSearchBoxInactive">
        <span class="left">
          <img id="MSearchSelect" src="search/mag_sel.png"
               onmouseover="return searchBox.OnSearchSelectShow()"
               onmouseout="return searchBox.OnSearchSelectHide()"
               alt=""/>
          <input type="text" id="MSearchField" value="Search" accesskey="S"
               onfocus="searchBox.OnSearchFieldFocus(true)" 
               onblur="searchBox.OnSearchFieldFocus(false)" 
               onkeyup="searchBox.OnSearchFieldChange(event)"/>
          </span><span class="right">
            <a id="MSearchClose" href="javascript:searchBox.CloseResultsWindow()"><img id="MSearchCloseImg" border="0" src="search/close.png" alt=""/></a>
          </span>
        </div>
      </li>
    </ul>
  </div>
  <div id="navrow2" class="tabs2">
    <ul class="tablist">
      <li><a href="files.html"><span>File&#160;List</span></a></li>
      <li><a href="globals.html"><span>File&#160;Members</span></a></li>
    </ul>
  </div>
</div>
<div class="header">
  <div class="summary">
<a href="#func-members">Functions</a>  </div>
  <div class="headertitle">
<div class="title">Host.c File Reference</div>  </div>
</div><!--header-->
<div class="contents">

<p>This contains the host functions for data allocations, message passing and host-side computations.  
<a href="#details">More...</a></p>
<div class="textblock"><code>#include &lt;string.h&gt;</code><br/>
<code>#include &lt;math.h&gt;</code><br/>
<code>#include &quot;<a class="el" href="Jacobi_8h.html">Jacobi.h</a>&quot;</code><br/>
</div><div class="textblock"><div class="dynheader">
Include dependency graph for Host.c:</div>
<div class="dyncontent">
<div class="center"><img src="Host_8c__incl.png" border="0" usemap="#Host_8c" alt=""/></div>
<map name="Host_8c" id="Host_8c">
<area shape="rect" id="node7" href="Jacobi_8h.html" title="The header containing the most relevant functions for the Jacobi solver." alt="" coords="179,83,248,112"/></map>
</div>
</div><table class="memberdecls">
<tr><td colspan="2"><h2><a name="func-members"></a>
Functions</h2></td></tr>
<tr><td class="memItemLeft" align="right" valign="top">void&#160;</td><td class="memItemRight" valign="bottom"><a class="el" href="Host_8c.html#a0ea9f99f85b380e912940f351366a862">Initialize</a> (int *argc, char ***argv, int *rank, int *size)</td></tr>
<tr><td class="mdescLeft">&#160;</td><td class="mdescRight">Initialize the MPI environment, allowing the CUDA device to be selected before (if necessary)  <a href="#a0ea9f99f85b380e912940f351366a862"></a><br/></td></tr>
<tr><td class="memItemLeft" align="right" valign="top">void&#160;</td><td class="memItemRight" valign="bottom"><a class="el" href="Host_8c.html#ad25cf65f450abbc68de1a6409b0c92ba">Finalize</a> (real *devBlocks[2], real *devSideEdges[2], real *devHaloLines[2], real *hostSendLines[2], real *hostRecvLines[2], real *devResidue, cudaStream_t copyStream)</td></tr>
<tr><td class="mdescLeft">&#160;</td><td class="mdescRight">Close (finalize) the MPI environment and deallocate buffers.  <a href="#ad25cf65f450abbc68de1a6409b0c92ba"></a><br/></td></tr>
<tr><td class="memItemLeft" align="right" valign="top">int&#160;</td><td class="memItemRight" valign="bottom"><a class="el" href="Host_8c.html#aef3118f365ec96030a022fa17b73fc42">ApplyTopology</a> (int *rank, int size, const int2 *topSize, int *neighbors, int2 *topIndex, MPI_Comm *cartComm)</td></tr>
<tr><td class="mdescLeft">&#160;</td><td class="mdescRight">Generates the 2D topology and establishes the neighbor relationships between MPI processes.  <a href="#aef3118f365ec96030a022fa17b73fc42"></a><br/></td></tr>
<tr><td class="memItemLeft" align="right" valign="top">void&#160;</td><td class="memItemRight" valign="bottom"><a class="el" href="Host_8c.html#a63442a950638ea4accb5dcb5ad7a8b87">InitializeDataChunk</a> (int topSizeY, int topIdxY, const int2 *domSize, const int *neighbors, cudaStream_t *copyStream, real *devBlocks[2], real *devSideEdges[2], real *devHaloLines[2], real *hostSendLines[2], real *hostRecvLines[2], real **devResidue)</td></tr>
<tr><td class="mdescLeft">&#160;</td><td class="mdescRight">This allocates and initializes all the relevant data buffers before the Jacobi run.  <a href="#a63442a950638ea4accb5dcb5ad7a8b87"></a><br/></td></tr>
<tr><td class="memItemLeft" align="right" valign="top">void&#160;</td><td class="memItemRight" valign="bottom"><a class="el" href="Host_8c.html#a053b144f9d3b11ad9b3563447876629f">PreRunJacobi</a> (MPI_Comm cartComm, int rank, int size, double *timerStart)</td></tr>
<tr><td class="mdescLeft">&#160;</td><td class="mdescRight">This function is called immediately before the main Jacobi loop.  <a href="#a053b144f9d3b11ad9b3563447876629f"></a><br/></td></tr>
<tr><td class="memItemLeft" align="right" valign="top">void&#160;</td><td class="memItemRight" valign="bottom"><a class="el" href="Host_8c.html#a29316e4a0e49080875975ead1809fdbb">PostRunJacobi</a> (MPI_Comm cartComm, int rank, int size, const int2 *topSize, const int2 *domSize, int iterations, int useFastSwap, double timerStart, double avgTransferTime)</td></tr>
<tr><td class="mdescLeft">&#160;</td><td class="mdescRight">This function is called immediately after the main Jacobi loop.  <a href="#a29316e4a0e49080875975ead1809fdbb"></a><br/></td></tr>
<tr><td class="memItemLeft" align="right" valign="top">double&#160;</td><td class="memItemRight" valign="bottom"><a class="el" href="Host_8c.html#ac28696e075d2053b4de5662556c16f3d">TransferAllHalos</a> (MPI_Comm cartComm, const int2 *domSize, const int2 *topIndex, const int *neighbors, cudaStream_t copyStream, real *devBlocks[2], real *devSideEdges[2], real *devHaloLines[2], real *hostSendLines[2], real *hostRecvLines[2])</td></tr>
<tr><td class="mdescLeft">&#160;</td><td class="mdescRight">This performs the exchanging of all necessary halos between 2 neighboring MPI processes.  <a href="#ac28696e075d2053b4de5662556c16f3d"></a><br/></td></tr>
<tr><td class="memItemLeft" align="right" valign="top">void&#160;</td><td class="memItemRight" valign="bottom"><a class="el" href="Host_8c.html#a52ea6f92f186de790a55fdf700026dd9">RunJacobi</a> (MPI_Comm cartComm, int rank, int size, const int2 *domSize, const int2 *topIndex, const int *neighbors, int useFastSwap, real *devBlocks[2], real *devSideEdges[2], real *devHaloLines[2], real *hostSendLines[2], real *hostRecvLines[2], real *devResidue, cudaStream_t copyStream, int *iterations, double *avgTransferTime)</td></tr>
<tr><td class="mdescLeft">&#160;</td><td class="mdescRight">This is the main Jacobi loop, which handles device computation and data exchange between MPI processes.  <a href="#a52ea6f92f186de790a55fdf700026dd9"></a><br/></td></tr>
</table>
<hr/><a name="details" id="details"></a><h2>Detailed Description</h2>
<div class="textblock"><p>This contains the host functions for data allocations, message passing and host-side computations. </p>
</div><hr/><h2>Function Documentation</h2>
<a class="anchor" id="aef3118f365ec96030a022fa17b73fc42"></a><!-- doxytag: member="Host.c::ApplyTopology" ref="aef3118f365ec96030a022fa17b73fc42" args="(int *rank, int size, const int2 *topSize, int *neighbors, int2 *topIndex, MPI_Comm *cartComm)" -->
<div class="memitem">
<div class="memproto">
      <table class="memname">
        <tr>
          <td class="memname">int <a class="el" href="Jacobi_8h.html#aef3118f365ec96030a022fa17b73fc42">ApplyTopology</a> </td>
          <td>(</td>
          <td class="paramtype">int *&#160;</td>
          <td class="paramname"><em>rank</em>, </td>
        </tr>
        <tr>
          <td class="paramkey"></td>
          <td></td>
          <td class="paramtype">int&#160;</td>
          <td class="paramname"><em>size</em>, </td>
        </tr>
        <tr>
          <td class="paramkey"></td>
          <td></td>
          <td class="paramtype">const int2 *&#160;</td>
          <td class="paramname"><em>topSize</em>, </td>
        </tr>
        <tr>
          <td class="paramkey"></td>
          <td></td>
          <td class="paramtype">int *&#160;</td>
          <td class="paramname"><em>neighbors</em>, </td>
        </tr>
        <tr>
          <td class="paramkey"></td>
          <td></td>
          <td class="paramtype">int2 *&#160;</td>
          <td class="paramname"><em>topIndex</em>, </td>
        </tr>
        <tr>
          <td class="paramkey"></td>
          <td></td>
          <td class="paramtype">MPI_Comm *&#160;</td>
          <td class="paramname"><em>cartComm</em>&#160;</td>
        </tr>
        <tr>
          <td></td>
          <td>)</td>
          <td></td><td></td>
        </tr>
      </table>
</div>
<div class="memdoc">

<p>Generates the 2D topology and establishes the neighbor relationships between MPI processes. </p>
<dl class="params"><dt><b>Parameters:</b></dt><dd>
  <table class="params">
    <tr><td class="paramdir">[in,out]</td><td class="paramname">rank</td><td>The rank of the calling MPI process </td></tr>
    <tr><td class="paramdir">[in]</td><td class="paramname">size</td><td>The total number of MPI processes available </td></tr>
    <tr><td class="paramdir">[in]</td><td class="paramname">topSize</td><td>The desired topology size (this must match the number of available MPI processes) </td></tr>
    <tr><td class="paramdir">[out]</td><td class="paramname">neighbors</td><td>The list that will be populated with the direct neighbors of the calling MPI process </td></tr>
    <tr><td class="paramdir">[out]</td><td class="paramname">topIndex</td><td>The 2D index that the calling MPI process will have in the topology </td></tr>
    <tr><td class="paramdir">[out]</td><td class="paramname">cartComm</td><td>The carthesian MPI communicator </td></tr>
  </table>
  </dd>
</dl>

<p><div class="dynheader">
Here is the call graph for this function:</div>
<div class="dyncontent">
<div class="center"><img src="Host_8c_aef3118f365ec96030a022fa17b73fc42_cgraph.png" border="0" usemap="#Host_8c_aef3118f365ec96030a022fa17b73fc42_cgraph" alt=""/></div>
<map name="Host_8c_aef3118f365ec96030a022fa17b73fc42_cgraph" id="Host_8c_aef3118f365ec96030a022fa17b73fc42_cgraph">
<area shape="rect" id="node3" href="CUDA__Aware__MPI_8c.html#aa8cb031e8cba7840a96582f530283fd5" title="This allows the MPI process to set the CUDA device after the MPI environment is initialized For the C..." alt="" coords="164,5,292,35"/></map>
</div>
</p>

</div>
</div>
<a class="anchor" id="ad25cf65f450abbc68de1a6409b0c92ba"></a><!-- doxytag: member="Host.c::Finalize" ref="ad25cf65f450abbc68de1a6409b0c92ba" args="(real *devBlocks[2], real *devSideEdges[2], real *devHaloLines[2], real *hostSendLines[2], real *hostRecvLines[2], real *devResidue, cudaStream_t copyStream)" -->
<div class="memitem">
<div class="memproto">
      <table class="memname">
        <tr>
          <td class="memname">void <a class="el" href="Jacobi_8h.html#ad25cf65f450abbc68de1a6409b0c92ba">Finalize</a> </td>
          <td>(</td>
          <td class="paramtype">real *&#160;</td>
          <td class="paramname"><em>devBlocks</em>[2], </td>
        </tr>
        <tr>
          <td class="paramkey"></td>
          <td></td>
          <td class="paramtype">real *&#160;</td>
          <td class="paramname"><em>devSideEdges</em>[2], </td>
        </tr>
        <tr>
          <td class="paramkey"></td>
          <td></td>
          <td class="paramtype">real *&#160;</td>
          <td class="paramname"><em>devHaloLines</em>[2], </td>
        </tr>
        <tr>
          <td class="paramkey"></td>
          <td></td>
          <td class="paramtype">real *&#160;</td>
          <td class="paramname"><em>hostSendLines</em>[2], </td>
        </tr>
        <tr>
          <td class="paramkey"></td>
          <td></td>
          <td class="paramtype">real *&#160;</td>
          <td class="paramname"><em>hostRecvLines</em>[2], </td>
        </tr>
        <tr>
          <td class="paramkey"></td>
          <td></td>
          <td class="paramtype">real *&#160;</td>
          <td class="paramname"><em>devResidue</em>, </td>
        </tr>
        <tr>
          <td class="paramkey"></td>
          <td></td>
          <td class="paramtype">cudaStream_t&#160;</td>
          <td class="paramname"><em>copyStream</em>&#160;</td>
        </tr>
        <tr>
          <td></td>
          <td>)</td>
          <td></td><td></td>
        </tr>
      </table>
</div>
<div class="memdoc">

<p>Close (finalize) the MPI environment and deallocate buffers. </p>
<dl class="params"><dt><b>Parameters:</b></dt><dd>
  <table class="params">
    <tr><td class="paramdir">[in]</td><td class="paramname">devBlocks</td><td>The 2 device blocks that were used during the Jacobi iterations </td></tr>
    <tr><td class="paramdir">[in]</td><td class="paramname">devSideEdges</td><td>The 2 device side edges that were used to hold updated halos before sending </td></tr>
    <tr><td class="paramdir">[in]</td><td class="paramname">devHaloLines</td><td>The 2 device lines that were used to hold received halos </td></tr>
    <tr><td class="paramdir">[in]</td><td class="paramname">hostSendLines</td><td>The 2 host send buffers that were used at halo exchange in the normal CUDA &amp; MPI version </td></tr>
    <tr><td class="paramdir">[in]</td><td class="paramname">hostRecvLines</td><td>The 2 host receive buffers that were used at halo exchange in the normal CUDA &amp; MPI version </td></tr>
    <tr><td class="paramdir">[in]</td><td class="paramname">devResidue</td><td>The global residue, kept in device memory </td></tr>
    <tr><td class="paramdir">[in]</td><td class="paramname">copyStream</td><td>The stream used to overlap top &amp; bottom halo exchange with side halo copy to host memory </td></tr>
  </table>
  </dd>
</dl>

</div>
</div>
<a class="anchor" id="a0ea9f99f85b380e912940f351366a862"></a><!-- doxytag: member="Host.c::Initialize" ref="a0ea9f99f85b380e912940f351366a862" args="(int *argc, char ***argv, int *rank, int *size)" -->
<div class="memitem">
<div class="memproto">
      <table class="memname">
        <tr>
          <td class="memname">void <a class="el" href="Jacobi_8h.html#a0ea9f99f85b380e912940f351366a862">Initialize</a> </td>
          <td>(</td>
          <td class="paramtype">int *&#160;</td>
          <td class="paramname"><em>argc</em>, </td>
        </tr>
        <tr>
          <td class="paramkey"></td>
          <td></td>
          <td class="paramtype">char ***&#160;</td>
          <td class="paramname"><em>argv</em>, </td>
        </tr>
        <tr>
          <td class="paramkey"></td>
          <td></td>
          <td class="paramtype">int *&#160;</td>
          <td class="paramname"><em>rank</em>, </td>
        </tr>
        <tr>
          <td class="paramkey"></td>
          <td></td>
          <td class="paramtype">int *&#160;</td>
          <td class="paramname"><em>size</em>&#160;</td>
        </tr>
        <tr>
          <td></td>
          <td>)</td>
          <td></td><td></td>
        </tr>
      </table>
</div>
<div class="memdoc">

<p>Initialize the MPI environment, allowing the CUDA device to be selected before (if necessary) </p>
<dl class="params"><dt><b>Parameters:</b></dt><dd>
  <table class="params">
    <tr><td class="paramdir">[in,out]</td><td class="paramname">argc</td><td>The number of command-line arguments </td></tr>
    <tr><td class="paramdir">[in,out]</td><td class="paramname">argv</td><td>The list of command-line arguments </td></tr>
    <tr><td class="paramdir">[out]</td><td class="paramname">rank</td><td>The global rank of the current MPI process </td></tr>
    <tr><td class="paramdir">[out]</td><td class="paramname">size</td><td>The total number of MPI processes available </td></tr>
  </table>
  </dd>
</dl>

<p><div class="dynheader">
Here is the call graph for this function:</div>
<div class="dyncontent">
<div class="center"><img src="Host_8c_a0ea9f99f85b380e912940f351366a862_cgraph.png" border="0" usemap="#Host_8c_a0ea9f99f85b380e912940f351366a862_cgraph" alt=""/></div>
<map name="Host_8c_a0ea9f99f85b380e912940f351366a862_cgraph" id="Host_8c_a0ea9f99f85b380e912940f351366a862_cgraph">
<area shape="rect" id="node3" href="CUDA__Aware__MPI_8c.html#ac7925a5469b04c01679b20577671c975" title="This allows the MPI process to set the CUDA device before the MPI environment is initialized For the ..." alt="" coords="125,5,264,35"/></map>
</div>
</p>

</div>
</div>
<a class="anchor" id="a63442a950638ea4accb5dcb5ad7a8b87"></a><!-- doxytag: member="Host.c::InitializeDataChunk" ref="a63442a950638ea4accb5dcb5ad7a8b87" args="(int topSizeY, int topIdxY, const int2 *domSize, const int *neighbors, cudaStream_t *copyStream, real *devBlocks[2], real *devSideEdges[2], real *devHaloLines[2], real *hostSendLines[2], real *hostRecvLines[2], real **devResidue)" -->
<div class="memitem">
<div class="memproto">
      <table class="memname">
        <tr>
          <td class="memname">void <a class="el" href="Jacobi_8h.html#a63442a950638ea4accb5dcb5ad7a8b87">InitializeDataChunk</a> </td>
          <td>(</td>
          <td class="paramtype">int&#160;</td>
          <td class="paramname"><em>topSizeY</em>, </td>
        </tr>
        <tr>
          <td class="paramkey"></td>
          <td></td>
          <td class="paramtype">int&#160;</td>
          <td class="paramname"><em>topIdxY</em>, </td>
        </tr>
        <tr>
          <td class="paramkey"></td>
          <td></td>
          <td class="paramtype">const int2 *&#160;</td>
          <td class="paramname"><em>domSize</em>, </td>
        </tr>
        <tr>
          <td class="paramkey"></td>
          <td></td>
          <td class="paramtype">const int *&#160;</td>
          <td class="paramname"><em>neighbors</em>, </td>
        </tr>
        <tr>
          <td class="paramkey"></td>
          <td></td>
          <td class="paramtype">cudaStream_t *&#160;</td>
          <td class="paramname"><em>copyStream</em>, </td>
        </tr>
        <tr>
          <td class="paramkey"></td>
          <td></td>
          <td class="paramtype">real *&#160;</td>
          <td class="paramname"><em>devBlocks</em>[2], </td>
        </tr>
        <tr>
          <td class="paramkey"></td>
          <td></td>
          <td class="paramtype">real *&#160;</td>
          <td class="paramname"><em>devSideEdges</em>[2], </td>
        </tr>
        <tr>
          <td class="paramkey"></td>
          <td></td>
          <td class="paramtype">real *&#160;</td>
          <td class="paramname"><em>devHaloLines</em>[2], </td>
        </tr>
        <tr>
          <td class="paramkey"></td>
          <td></td>
          <td class="paramtype">real *&#160;</td>
          <td class="paramname"><em>hostSendLines</em>[2], </td>
        </tr>
        <tr>
          <td class="paramkey"></td>
          <td></td>
          <td class="paramtype">real *&#160;</td>
          <td class="paramname"><em>hostRecvLines</em>[2], </td>
        </tr>
        <tr>
          <td class="paramkey"></td>
          <td></td>
          <td class="paramtype">real **&#160;</td>
          <td class="paramname"><em>devResidue</em>&#160;</td>
        </tr>
        <tr>
          <td></td>
          <td>)</td>
          <td></td><td></td>
        </tr>
      </table>
</div>
<div class="memdoc">

<p>This allocates and initializes all the relevant data buffers before the Jacobi run. </p>
<dl class="params"><dt><b>Parameters:</b></dt><dd>
  <table class="params">
    <tr><td class="paramdir">[in]</td><td class="paramname">topSizeY</td><td>The size of the topology in the Y direction </td></tr>
    <tr><td class="paramdir">[in]</td><td class="paramname">topIdxY</td><td>The Y index of the calling MPI process in the topology </td></tr>
    <tr><td class="paramdir">[in]</td><td class="paramname">domSize</td><td>The size of the local domain (for which only the current MPI process is responsible) </td></tr>
    <tr><td class="paramdir">[in]</td><td class="paramname">neighbors</td><td>The neighbor ranks, according to the topology </td></tr>
    <tr><td class="paramdir">[in]</td><td class="paramname">copyStream</td><td>The stream used to overlap top &amp; bottom halo exchange with side halo copy to host memory </td></tr>
    <tr><td class="paramdir">[out]</td><td class="paramname">devBlocks</td><td>The 2 device blocks that will be updated during the Jacobi run </td></tr>
    <tr><td class="paramdir">[out]</td><td class="paramname">devSideEdges</td><td>The 2 side edges (parallel to the Y direction) that will hold the packed halo values before sending them </td></tr>
    <tr><td class="paramdir">[out]</td><td class="paramname">devHaloLines</td><td>The 2 halo lines (parallel to the Y direction) that will hold the packed halo values after receiving them </td></tr>
    <tr><td class="paramdir">[out]</td><td class="paramname">hostSendLines</td><td>The 2 host send buffers that will be used during the halo exchange by the normal CUDA &amp; MPI version </td></tr>
    <tr><td class="paramdir">[out]</td><td class="paramname">hostRecvLines</td><td>The 2 host receive buffers that will be used during the halo exchange by the normal CUDA &amp; MPI version </td></tr>
    <tr><td class="paramdir">[out]</td><td class="paramname">devResidue</td><td>The global device residue, which will be updated after every Jacobi iteration </td></tr>
  </table>
  </dd>
</dl>

</div>
</div>
<a class="anchor" id="a29316e4a0e49080875975ead1809fdbb"></a><!-- doxytag: member="Host.c::PostRunJacobi" ref="a29316e4a0e49080875975ead1809fdbb" args="(MPI_Comm cartComm, int rank, int size, const int2 *topSize, const int2 *domSize, int iterations, int useFastSwap, double timerStart, double avgTransferTime)" -->
<div class="memitem">
<div class="memproto">
      <table class="memname">
        <tr>
          <td class="memname">void <a class="el" href="Jacobi_8h.html#a29316e4a0e49080875975ead1809fdbb">PostRunJacobi</a> </td>
          <td>(</td>
          <td class="paramtype">MPI_Comm&#160;</td>
          <td class="paramname"><em>cartComm</em>, </td>
        </tr>
        <tr>
          <td class="paramkey"></td>
          <td></td>
          <td class="paramtype">int&#160;</td>
          <td class="paramname"><em>rank</em>, </td>
        </tr>
        <tr>
          <td class="paramkey"></td>
          <td></td>
          <td class="paramtype">int&#160;</td>
          <td class="paramname"><em>size</em>, </td>
        </tr>
        <tr>
          <td class="paramkey"></td>
          <td></td>
          <td class="paramtype">const int2 *&#160;</td>
          <td class="paramname"><em>topSize</em>, </td>
        </tr>
        <tr>
          <td class="paramkey"></td>
          <td></td>
          <td class="paramtype">const int2 *&#160;</td>
          <td class="paramname"><em>domSize</em>, </td>
        </tr>
        <tr>
          <td class="paramkey"></td>
          <td></td>
          <td class="paramtype">int&#160;</td>
          <td class="paramname"><em>iterations</em>, </td>
        </tr>
        <tr>
          <td class="paramkey"></td>
          <td></td>
          <td class="paramtype">int&#160;</td>
          <td class="paramname"><em>useFastSwap</em>, </td>
        </tr>
        <tr>
          <td class="paramkey"></td>
          <td></td>
          <td class="paramtype">double&#160;</td>
          <td class="paramname"><em>timerStart</em>, </td>
        </tr>
        <tr>
          <td class="paramkey"></td>
          <td></td>
          <td class="paramtype">double&#160;</td>
          <td class="paramname"><em>avgTransferTime</em>&#160;</td>
        </tr>
        <tr>
          <td></td>
          <td>)</td>
          <td></td><td></td>
        </tr>
      </table>
</div>
<div class="memdoc">

<p>This function is called immediately after the main Jacobi loop. </p>
<dl class="params"><dt><b>Parameters:</b></dt><dd>
  <table class="params">
    <tr><td class="paramdir">[in]</td><td class="paramname">cartComm</td><td>The carthesian communicator </td></tr>
    <tr><td class="paramdir">[in]</td><td class="paramname">rank</td><td>The rank of the calling MPI process </td></tr>
    <tr><td class="paramdir">[in]</td><td class="paramname">topSize</td><td>The size of the topology </td></tr>
    <tr><td class="paramdir">[in]</td><td class="paramname">domSize</td><td>The size of the local domain </td></tr>
    <tr><td class="paramdir">[in]</td><td class="paramname">iterations</td><td>The number of successfully completed Jacobi iterations </td></tr>
    <tr><td class="paramdir">[in]</td><td class="paramname">useFastSwap</td><td>The flag indicating if fast pointer swapping was used to exchange blocks </td></tr>
    <tr><td class="paramdir">[in]</td><td class="paramname">timerStart</td><td>The Jacobi loop starting moment (measured as wall-time) </td></tr>
    <tr><td class="paramdir">[in]</td><td class="paramname">avgTransferTime</td><td>The average time spent performing MPI transfers (per process) </td></tr>
  </table>
  </dd>
</dl>

</div>
</div>
<a class="anchor" id="a053b144f9d3b11ad9b3563447876629f"></a><!-- doxytag: member="Host.c::PreRunJacobi" ref="a053b144f9d3b11ad9b3563447876629f" args="(MPI_Comm cartComm, int rank, int size, double *timerStart)" -->
<div class="memitem">
<div class="memproto">
      <table class="memname">
        <tr>
          <td class="memname">void <a class="el" href="Jacobi_8h.html#a053b144f9d3b11ad9b3563447876629f">PreRunJacobi</a> </td>
          <td>(</td>
          <td class="paramtype">MPI_Comm&#160;</td>
          <td class="paramname"><em>cartComm</em>, </td>
        </tr>
        <tr>
          <td class="paramkey"></td>
          <td></td>
          <td class="paramtype">int&#160;</td>
          <td class="paramname"><em>rank</em>, </td>
        </tr>
        <tr>
          <td class="paramkey"></td>
          <td></td>
          <td class="paramtype">int&#160;</td>
          <td class="paramname"><em>size</em>, </td>
        </tr>
        <tr>
          <td class="paramkey"></td>
          <td></td>
          <td class="paramtype">double *&#160;</td>
          <td class="paramname"><em>timerStart</em>&#160;</td>
        </tr>
        <tr>
          <td></td>
          <td>)</td>
          <td></td><td></td>
        </tr>
      </table>
</div>
<div class="memdoc">

<p>This function is called immediately before the main Jacobi loop. </p>
<dl class="params"><dt><b>Parameters:</b></dt><dd>
  <table class="params">
    <tr><td class="paramdir">[in]</td><td class="paramname">cartComm</td><td>The carthesian communicator </td></tr>
    <tr><td class="paramdir">[in]</td><td class="paramname">rank</td><td>The rank of the calling MPI process </td></tr>
    <tr><td class="paramdir">[in]</td><td class="paramname">size</td><td>The total number of MPI processes available </td></tr>
    <tr><td class="paramdir">[out]</td><td class="paramname">timerStart</td><td>The Jacobi loop starting moment (measured as wall-time) </td></tr>
  </table>
  </dd>
</dl>

</div>
</div>
<a class="anchor" id="a52ea6f92f186de790a55fdf700026dd9"></a><!-- doxytag: member="Host.c::RunJacobi" ref="a52ea6f92f186de790a55fdf700026dd9" args="(MPI_Comm cartComm, int rank, int size, const int2 *domSize, const int2 *topIndex, const int *neighbors, int useFastSwap, real *devBlocks[2], real *devSideEdges[2], real *devHaloLines[2], real *hostSendLines[2], real *hostRecvLines[2], real *devResidue, cudaStream_t copyStream, int *iterations, double *avgTransferTime)" -->
<div class="memitem">
<div class="memproto">
      <table class="memname">
        <tr>
          <td class="memname">void <a class="el" href="Jacobi_8h.html#a52ea6f92f186de790a55fdf700026dd9">RunJacobi</a> </td>
          <td>(</td>
          <td class="paramtype">MPI_Comm&#160;</td>
          <td class="paramname"><em>cartComm</em>, </td>
        </tr>
        <tr>
          <td class="paramkey"></td>
          <td></td>
          <td class="paramtype">int&#160;</td>
          <td class="paramname"><em>rank</em>, </td>
        </tr>
        <tr>
          <td class="paramkey"></td>
          <td></td>
          <td class="paramtype">int&#160;</td>
          <td class="paramname"><em>size</em>, </td>
        </tr>
        <tr>
          <td class="paramkey"></td>
          <td></td>
          <td class="paramtype">const int2 *&#160;</td>
          <td class="paramname"><em>domSize</em>, </td>
        </tr>
        <tr>
          <td class="paramkey"></td>
          <td></td>
          <td class="paramtype">const int2 *&#160;</td>
          <td class="paramname"><em>topIndex</em>, </td>
        </tr>
        <tr>
          <td class="paramkey"></td>
          <td></td>
          <td class="paramtype">const int *&#160;</td>
          <td class="paramname"><em>neighbors</em>, </td>
        </tr>
        <tr>
          <td class="paramkey"></td>
          <td></td>
          <td class="paramtype">int&#160;</td>
          <td class="paramname"><em>useFastSwap</em>, </td>
        </tr>
        <tr>
          <td class="paramkey"></td>
          <td></td>
          <td class="paramtype">real *&#160;</td>
          <td class="paramname"><em>devBlocks</em>[2], </td>
        </tr>
        <tr>
          <td class="paramkey"></td>
          <td></td>
          <td class="paramtype">real *&#160;</td>
          <td class="paramname"><em>devSideEdges</em>[2], </td>
        </tr>
        <tr>
          <td class="paramkey"></td>
          <td></td>
          <td class="paramtype">real *&#160;</td>
          <td class="paramname"><em>devHaloLines</em>[2], </td>
        </tr>
        <tr>
          <td class="paramkey"></td>
          <td></td>
          <td class="paramtype">real *&#160;</td>
          <td class="paramname"><em>hostSendLines</em>[2], </td>
        </tr>
        <tr>
          <td class="paramkey"></td>
          <td></td>
          <td class="paramtype">real *&#160;</td>
          <td class="paramname"><em>hostRecvLines</em>[2], </td>
        </tr>
        <tr>
          <td class="paramkey"></td>
          <td></td>
          <td class="paramtype">real *&#160;</td>
          <td class="paramname"><em>devResidue</em>, </td>
        </tr>
        <tr>
          <td class="paramkey"></td>
          <td></td>
          <td class="paramtype">cudaStream_t&#160;</td>
          <td class="paramname"><em>copyStream</em>, </td>
        </tr>
        <tr>
          <td class="paramkey"></td>
          <td></td>
          <td class="paramtype">int *&#160;</td>
          <td class="paramname"><em>iterations</em>, </td>
        </tr>
        <tr>
          <td class="paramkey"></td>
          <td></td>
          <td class="paramtype">double *&#160;</td>
          <td class="paramname"><em>avgTransferTime</em>&#160;</td>
        </tr>
        <tr>
          <td></td>
          <td>)</td>
          <td></td><td></td>
        </tr>
      </table>
</div>
<div class="memdoc">

<p>This is the main Jacobi loop, which handles device computation and data exchange between MPI processes. </p>
<dl class="params"><dt><b>Parameters:</b></dt><dd>
  <table class="params">
    <tr><td class="paramdir">[in]</td><td class="paramname">cartComm</td><td>The carthesian MPI communicator </td></tr>
    <tr><td class="paramdir">[in]</td><td class="paramname">rank</td><td>The rank of the calling MPI process </td></tr>
    <tr><td class="paramdir">[in]</td><td class="paramname">size</td><td>The number of available MPI processes </td></tr>
    <tr><td class="paramdir">[in]</td><td class="paramname">domSize</td><td>The 2D size of the local domain </td></tr>
    <tr><td class="paramdir">[in]</td><td class="paramname">topIndex</td><td>The 2D index of the calling MPI process in the topology </td></tr>
    <tr><td class="paramdir">[in]</td><td class="paramname">neighbors</td><td>The list of ranks which are direct neighbors to the caller </td></tr>
    <tr><td class="paramdir">[in]</td><td class="paramname">useFastSwap</td><td>This flag indicates if blocks should be swapped through pointer copy (faster) or through element-by-element copy (slower) </td></tr>
    <tr><td class="paramdir">[in,out]</td><td class="paramname">devBlocks</td><td>The 2 device blocks that are updated during the Jacobi run </td></tr>
    <tr><td class="paramdir">[in,out]</td><td class="paramname">devSideEdges</td><td>The 2 side edges (parallel to the Y direction) that hold the packed halo values before sending them </td></tr>
    <tr><td class="paramdir">[in,out]</td><td class="paramname">devHaloLines</td><td>The 2 halo lines (parallel to the Y direction) that hold the packed halo values after receiving them </td></tr>
    <tr><td class="paramdir">[in,out]</td><td class="paramname">hostSendLines</td><td>The 2 host send buffers that are used during the halo exchange by the normal CUDA &amp; MPI version </td></tr>
    <tr><td class="paramdir">[in,out]</td><td class="paramname">hostRecvLines</td><td>The 2 host receive buffers that are used during the halo exchange by the normal CUDA &amp; MPI version </td></tr>
    <tr><td class="paramdir">[in,out]</td><td class="paramname">devResidue</td><td>The global device residue, which gets updated after every Jacobi iteration </td></tr>
    <tr><td class="paramdir">[in]</td><td class="paramname">copyStream</td><td>The stream used to overlap top &amp; bottom halo exchange with side halo copy to host memory </td></tr>
    <tr><td class="paramdir">[out]</td><td class="paramname">iterations</td><td>The number of successfully completed iterations </td></tr>
    <tr><td class="paramdir">[out]</td><td class="paramname">avgTransferTime</td><td>The average time spent performing MPI transfers (per process) </td></tr>
  </table>
  </dd>
</dl>

<p><div class="dynheader">
Here is the call graph for this function:</div>
<div class="dyncontent">
<div class="center"><img src="Host_8c_a52ea6f92f186de790a55fdf700026dd9_cgraph.png" border="0" usemap="#Host_8c_a52ea6f92f186de790a55fdf700026dd9_cgraph" alt=""/></div>
<map name="Host_8c_a52ea6f92f186de790a55fdf700026dd9_cgraph" id="Host_8c_a52ea6f92f186de790a55fdf700026dd9_cgraph">
<area shape="rect" id="node3" href="Device_8cu.html#a08324d1c9b1ee7193242670ba4b97d88" title="The host wrapper for one Jacobi iteration." alt="" coords="143,5,263,35"/><area shape="rect" id="node7" href="Device_8cu.html#aeaebe085afbffaa0d6bcd77fa16ceb97" title="The host wrapper for copying the updated block over the old one, after a Jacobi iteration finishes..." alt="" coords="140,59,265,88"/><area shape="rect" id="node10" href="Host_8c.html#ac28696e075d2053b4de5662556c16f3d" title="This performs the exchanging of all necessary halos between 2 neighboring MPI processes." alt="" coords="144,137,261,167"/><area shape="rect" id="node5" href="Device_8cu.html#a87c5ce86d74764a06a13b73dff013870" title="The host function for checking the result of a CUDA API call." alt="" coords="565,83,680,112"/><area shape="rect" id="node12" href="Device_8cu.html#a95a52e3c2e58d2d94403a0dce2e29686" title="The host wrapper for copying (packing) the values on the left and right side of the data block to sep..." alt="" coords="315,108,517,137"/><area shape="rect" id="node15" href="CUDA__Aware__MPI_8c.html#a1dc0674ebcbca5a781885f2204d78251" title="Exchange halo values between 2 direct neighbors This is the main difference between the normal CUDA &amp;..." alt="" coords="359,161,473,191"/><area shape="rect" id="node17" href="Device_8cu.html#a3b641f8b5d521a230416b64d8b809324" title="The host wrapper for copying (unpacking) the values from the halo buffers to the left and right side ..." alt="" coords="337,215,495,244"/></map>
</div>
</p>

</div>
</div>
<a class="anchor" id="ac28696e075d2053b4de5662556c16f3d"></a><!-- doxytag: member="Host.c::TransferAllHalos" ref="ac28696e075d2053b4de5662556c16f3d" args="(MPI_Comm cartComm, const int2 *domSize, const int2 *topIndex, const int *neighbors, cudaStream_t copyStream, real *devBlocks[2], real *devSideEdges[2], real *devHaloLines[2], real *hostSendLines[2], real *hostRecvLines[2])" -->
<div class="memitem">
<div class="memproto">
      <table class="memname">
        <tr>
          <td class="memname">double <a class="el" href="Host_8c.html#ac28696e075d2053b4de5662556c16f3d">TransferAllHalos</a> </td>
          <td>(</td>
          <td class="paramtype">MPI_Comm&#160;</td>
          <td class="paramname"><em>cartComm</em>, </td>
        </tr>
        <tr>
          <td class="paramkey"></td>
          <td></td>
          <td class="paramtype">const int2 *&#160;</td>
          <td class="paramname"><em>domSize</em>, </td>
        </tr>
        <tr>
          <td class="paramkey"></td>
          <td></td>
          <td class="paramtype">const int2 *&#160;</td>
          <td class="paramname"><em>topIndex</em>, </td>
        </tr>
        <tr>
          <td class="paramkey"></td>
          <td></td>
          <td class="paramtype">const int *&#160;</td>
          <td class="paramname"><em>neighbors</em>, </td>
        </tr>
        <tr>
          <td class="paramkey"></td>
          <td></td>
          <td class="paramtype">cudaStream_t&#160;</td>
          <td class="paramname"><em>copyStream</em>, </td>
        </tr>
        <tr>
          <td class="paramkey"></td>
          <td></td>
          <td class="paramtype">real *&#160;</td>
          <td class="paramname"><em>devBlocks</em>[2], </td>
        </tr>
        <tr>
          <td class="paramkey"></td>
          <td></td>
          <td class="paramtype">real *&#160;</td>
          <td class="paramname"><em>devSideEdges</em>[2], </td>
        </tr>
        <tr>
          <td class="paramkey"></td>
          <td></td>
          <td class="paramtype">real *&#160;</td>
          <td class="paramname"><em>devHaloLines</em>[2], </td>
        </tr>
        <tr>
          <td class="paramkey"></td>
          <td></td>
          <td class="paramtype">real *&#160;</td>
          <td class="paramname"><em>hostSendLines</em>[2], </td>
        </tr>
        <tr>
          <td class="paramkey"></td>
          <td></td>
          <td class="paramtype">real *&#160;</td>
          <td class="paramname"><em>hostRecvLines</em>[2]&#160;</td>
        </tr>
        <tr>
          <td></td>
          <td>)</td>
          <td></td><td></td>
        </tr>
      </table>
</div>
<div class="memdoc">

<p>This performs the exchanging of all necessary halos between 2 neighboring MPI processes. </p>
<dl class="params"><dt><b>Parameters:</b></dt><dd>
  <table class="params">
    <tr><td class="paramdir">[in]</td><td class="paramname">cartComm</td><td>The carthesian MPI communicator </td></tr>
    <tr><td class="paramdir">[in]</td><td class="paramname">domSize</td><td>The 2D size of the local domain </td></tr>
    <tr><td class="paramdir">[in]</td><td class="paramname">topIndex</td><td>The 2D index of the calling MPI process in the topology </td></tr>
    <tr><td class="paramdir">[in]</td><td class="paramname">neighbors</td><td>The list of ranks which are direct neighbors to the caller </td></tr>
    <tr><td class="paramdir">[in]</td><td class="paramname">copyStream</td><td>The stream used to overlap top &amp; bottom halo exchange with side halo copy to host memory </td></tr>
    <tr><td class="paramdir">[in,out]</td><td class="paramname">devBlocks</td><td>The 2 device blocks that are updated during the Jacobi run </td></tr>
    <tr><td class="paramdir">[in,out]</td><td class="paramname">devSideEdges</td><td>The 2 side edges (parallel to the Y direction) that hold the packed halo values before sending them </td></tr>
    <tr><td class="paramdir">[in,out]</td><td class="paramname">devHaloLines</td><td>The 2 halo lines (parallel to the Y direction) that hold the packed halo values after receiving them </td></tr>
    <tr><td class="paramdir">[in,out]</td><td class="paramname">hostSendLines</td><td>The 2 host send buffers that are used during the halo exchange by the normal CUDA &amp; MPI version </td></tr>
    <tr><td class="paramdir">[in,out]</td><td class="paramname">hostRecvLines</td><td>The 2 host receive buffers that are used during the halo exchange by the normal CUDA &amp; MPI version </td></tr>
  </table>
  </dd>
</dl>
<dl class="return"><dt><b>Returns:</b></dt><dd>The time spent during the MPI transfers </dd></dl>

<p><div class="dynheader">
Here is the call graph for this function:</div>
<div class="dyncontent">
<div class="center"><img src="Host_8c_ac28696e075d2053b4de5662556c16f3d_cgraph.png" border="0" usemap="#Host_8c_ac28696e075d2053b4de5662556c16f3d_cgraph" alt=""/></div>
<map name="Host_8c_ac28696e075d2053b4de5662556c16f3d_cgraph" id="Host_8c_ac28696e075d2053b4de5662556c16f3d_cgraph">
<area shape="rect" id="node3" href="Device_8cu.html#a95a52e3c2e58d2d94403a0dce2e29686" title="The host wrapper for copying (packing) the values on the left and right side of the data block to sep..." alt="" coords="173,5,376,35"/><area shape="rect" id="node7" href="CUDA__Aware__MPI_8c.html#a1dc0674ebcbca5a781885f2204d78251" title="Exchange halo values between 2 direct neighbors This is the main difference between the normal CUDA &amp;..." alt="" coords="217,59,332,88"/><area shape="rect" id="node9" href="Device_8cu.html#a3b641f8b5d521a230416b64d8b809324" title="The host wrapper for copying (unpacking) the values from the halo buffers to the left and right side ..." alt="" coords="196,112,353,141"/><area shape="rect" id="node5" href="Device_8cu.html#a87c5ce86d74764a06a13b73dff013870" title="The host function for checking the result of a CUDA API call." alt="" coords="424,59,539,88"/></map>
</div>
</p>

</div>
</div>
</div><!-- contents -->
<!-- window showing the filter options -->
<div id="MSearchSelectWindow"
     onmouseover="return searchBox.OnSearchSelectShow()"
     onmouseout="return searchBox.OnSearchSelectHide()"
     onkeydown="return searchBox.OnSearchSelectKey(event)">
<a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(0)"><span class="SelectionMark">&#160;</span>All</a><a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(1)"><span class="SelectionMark">&#160;</span>Files</a><a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(2)"><span class="SelectionMark">&#160;</span>Functions</a><a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(3)"><span class="SelectionMark">&#160;</span>Defines</a></div>

<!-- iframe showing the search results (closed by default) -->
<div id="MSearchResultsWindow">
<iframe src="javascript:void(0)" frameborder="0" 
        name="MSearchResults" id="MSearchResults">
</iframe>
</div>



<hr class="footer"/><address class="footer"><small>
Generated on Fri Mar 1 2013 15:52:54 for Jacobi by &#160;<a href="http://www.doxygen.org/index.html">
<img class="footer" src="doxygen.png" alt="doxygen"/>
</a> 1.7.6.1
</small></address>

</body>
</html>
